#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
豆瓣电影Top 250爬虫
使用Python标准库爬取豆瓣电影Top 250的电影信息
"""

import urllib.request
import urllib.error
import html.parser
import csv
import time
import random
from urllib.parse import quote

class DoubanMovieParser(html.parser.HTMLParser):
    """豆瓣电影HTML解析器"""
    
    def __init__(self):
        super().__init__()
        self.movies = []
        self.current_movie = {}
        self.in_item = False
        self.in_info = False
        self.in_hd = False
        self.in_bd = False
        self.in_quote = False
        self.title_count = 0
        self.content_buffer = []
        self.recording = False
        self.record_tag = ''
        self.record_class = ''
        self.director_info = ''
        self.actor_info = ''
        self.year_info = ''
        self.country_info = ''
        self.genre_info = ''
        self.rating_info = ''
        self.quote_info = ''
    
    def handle_starttag(self, tag, attrs):
        # 将属性转换为字典便于查找
        attr_dict = dict(attrs)
        
        # 检测电影项的开始
        if tag == 'div' and attr_dict.get('class') == 'item':
            self.in_item = True
            self.current_movie = {}
            self.title_count = 0
        
        # 检测info部分
        elif self.in_item and tag == 'div' and attr_dict.get('class') == 'info':
            self.in_info = True
        
        # 检测hd部分（包含标题）
        elif self.in_info and tag == 'div' and attr_dict.get('class') == 'hd':
            self.in_hd = True
        
        # 检测标题
        elif self.in_hd and tag == 'span' and attr_dict.get('class') == 'title':
            self.recording = True
            self.record_tag = 'title'
        
        # 检测其他标题（别名）
        elif self.in_hd and tag == 'span' and attr_dict.get('class') == 'other':
            self.recording = True
            self.record_tag = 'other_title'
        
        # 检测bd部分（包含导演、主演、年份等信息）
        elif self.in_info and tag == 'div' and attr_dict.get('class') == 'bd':
            self.in_bd = True
        
        # 检测评分
        elif self.in_bd and tag == 'span' and attr_dict.get('class') == 'rating_num':
            self.recording = True
            self.record_tag = 'rating'
        
        # 检测评价人数
        elif self.in_bd and tag == 'span' and len(attr_dict) == 0:
            # 这是一个简单的判断，实际可能需要更精确
            self.recording = True
            self.record_tag = 'rating_people'
        
        # 检测引用（经典台词）
        elif self.in_bd and tag == 'span' and attr_dict.get('class') == 'inq':
            self.in_quote = True
            self.recording = True
            self.record_tag = 'quote'
        
        # 检测p标签（包含导演、主演、年份等信息）
        elif self.in_bd and tag == 'p':
            self.recording = True
            self.record_tag = 'movie_info'
    
    def handle_endtag(self, tag):
        # 处理标签结束
        if tag == 'div' and self.in_item and self.in_info and self.in_bd:
            self.in_bd = False
        
        elif tag == 'div' and self.in_item and self.in_info:
            self.in_info = False
        
        elif tag == 'div' and self.in_item:
            self.in_item = False
            # 当一个电影项结束时，将收集的数据添加到列表中
            if 'title' in self.current_movie:
                # 处理电影信息
                self._process_movie_info()
                self.movies.append(self.current_movie)
        
        elif tag == 'div' and self.in_hd:
            self.in_hd = False
        
        elif tag == 'p' and self.recording and self.record_tag == 'movie_info':
            self.recording = False
        
        elif tag == 'span' and self.recording:
            self.recording = False
            
            # 处理收集到的内容
            content = ''.join(self.content_buffer).strip()
            self.content_buffer = []
            
            if self.record_tag == 'title':
                if self.title_count == 0:
                    self.current_movie['title'] = content  # 中文标题
                else:
                    self.current_movie['original_title'] = content  # 英文标题
                self.title_count += 1
            elif self.record_tag == 'other_title':
                self.current_movie['other_title'] = content
            elif self.record_tag == 'rating':
                self.current_movie['rating'] = content
            elif self.record_tag == 'rating_people':
                self.current_movie['rating_people'] = content
            elif self.record_tag == 'quote':
                self.current_movie['quote'] = content
            elif self.record_tag == 'movie_info':
                self.current_movie['raw_info'] = content
    
    def handle_data(self, data):
        # 收集数据
        if self.recording:
            self.content_buffer.append(data)
    
    def _process_movie_info(self):
        """处理电影的详细信息"""
        if 'raw_info' in self.current_movie:
            info_text = self.current_movie['raw_info']
            lines = info_text.split('\n')
            
            # 处理第一行（导演和主演）
            if len(lines) > 0:
                first_line = lines[0].strip()
                if '导演:' in first_line:
                    # 分割导演和主演
                    parts = first_line.split('主演:')
                    if len(parts) > 0:
                        director_part = parts[0].replace('导演:', '').strip()
                        self.current_movie['director'] = director_part
                    if len(parts) > 1:
                        actor_part = parts[1].strip()
                        self.current_movie['actors'] = actor_part
                else:
                    self.current_movie['director'] = first_line
            
            # 处理第二行（年份、国家、类型）
            if len(lines) > 1:
                second_line = lines[1].strip()
                # 使用'/'分割信息
                infos = [item.strip() for item in second_line.split('/')]
                if len(infos) > 0:
                    self.current_movie['year'] = infos[0]
                if len(infos) > 1:
                    self.current_movie['country'] = infos[1]
                if len(infos) > 2:
                    self.current_movie['genre'] = infos[2]
    
    def get_movies(self):
        """返回解析到的电影列表"""
        return self.movies

def fetch_page(url):
    """获取网页内容"""
    # 添加User-Agent以模拟浏览器访问
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }
    
    request = urllib.request.Request(url, headers=headers)
    
    try:
        with urllib.request.urlopen(request, timeout=30) as response:
            # 读取响应内容
            content = response.read()
            # 尝试使用UTF-8解码
            return content.decode('utf-8')
    except urllib.error.URLError as e:
        print(f"获取页面失败: {url}")
        print(f"错误信息: {e}")
        return None
    except UnicodeDecodeError:
        # 如果UTF-8解码失败，尝试其他编码
        try:
            return content.decode('gbk')
        except UnicodeDecodeError:
            print(f"解码页面内容失败: {url}")
            return None

def crawl_douban_top250():
    """爬取豆瓣电影Top 250"""
    base_url = 'https://movie.douban.com/top250'
    all_movies = []
    
    print("开始爬取豆瓣电影Top 250...")
    
    # 豆瓣电影Top 250共有10页，每页25部电影
    for page in range(10):
        start = page * 25
        url = f"{base_url}?start={start}&filter="
        
        print(f"正在爬取第{page+1}页: {url}")
        
        # 获取页面内容
        html_content = fetch_page(url)
        
        if html_content:
            # 创建解析器并解析HTML
            parser = DoubanMovieParser()
            parser.feed(html_content)
            
            # 获取解析到的电影列表
            movies = parser.get_movies()
            all_movies.extend(movies)
            
            print(f"第{page+1}页解析完成，获取到{len(movies)}部电影")
            
            # 添加随机延时，避免请求过于频繁
            delay = random.uniform(1, 3)
            print(f"休息{delay:.2f}秒...")
            time.sleep(delay)
        else:
            print(f"第{page+1}页爬取失败，跳过")
    
    print(f"爬取完成！总共获取到{len(all_movies)}部电影")
    return all_movies

def save_to_csv(movies, filename='douban_top250.csv'):
    """将电影数据保存到CSV文件"""
    # 定义CSV文件的字段
    fields = ['排名', '电影名称', '原始名称', '别名', '导演', '主演', '年份', '国家', '类型', '评分', '评价人数', '经典台词']
    
    with open(filename, 'w', newline='', encoding='utf-8-sig') as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames=fields)
        
        # 写入表头
        writer.writeheader()
        
        # 写入数据
        for i, movie in enumerate(movies, 1):
            row = {
                '排名': i,
                '电影名称': movie.get('title', ''),
                '原始名称': movie.get('original_title', '').replace('/ ', ''),
                '别名': movie.get('other_title', '').replace('/ ', ''),
                '导演': movie.get('director', ''),
                '主演': movie.get('actors', ''),
                '年份': movie.get('year', ''),
                '国家': movie.get('country', ''),
                '类型': movie.get('genre', ''),
                '评分': movie.get('rating', ''),
                '评价人数': movie.get('rating_people', ''),
                '经典台词': movie.get('quote', '')
            }
            writer.writerow(row)
    
    print(f"数据已保存到文件: {filename}")

def main():
    """主函数"""
    try:
        # 爬取电影数据
        movies = crawl_douban_top250()
        
        if movies:
            # 保存到CSV文件
            csv_filename = 'douban_top250.csv'
            save_to_csv(movies, csv_filename)
            print(f"\n恭喜！豆瓣电影Top 250数据爬取完成并保存到{csv_filename}。")
            print("您可以使用Excel或其他电子表格软件打开此文件查看详细信息。")
        else:
            print("未能获取到电影数据，请检查网络连接或稍后再试。")
    
    except Exception as e:
        print(f"程序运行出错: {e}")
        import traceback
        traceback.print_exc()
    
    finally:
        # 等待用户输入，防止程序立即关闭
        input("\n按回车键退出程序...")

if __name__ == '__main__':
    main()